{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "7d044831-9a38-4eae-b676-d5d90a62f4f3",
   "metadata": {},
   "source": [
    "# 配置GPTQ算法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3592a507-dd56-4d89-a6a2-27df754dd64f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AutoModelForCausalLM, AutoTokenizer, GPTQConfig\n",
    "import torch\n",
    "\n",
    "quantization_config = GPTQConfig(\n",
    "     bits=2, # 量化精度\n",
    "     group_size=2,\n",
    "     dataset=\"wikitext2\",\n",
    "     desc_act=False,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e6193e7f-fa70-4f31-8cfc-8af38cfe9bd5",
   "metadata": {},
   "source": [
    "# 配置模型 facebook/opt-1.3b 因为云服务器的显存只有16G，使用不了OPT-6.7B"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4192055a-7e40-4fe3-baa3-3a811fc6d18b",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name_or_path = \"facebook/opt-1.3b\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1265612e-0927-444b-a2f1-8397e677b301",
   "metadata": {},
   "source": [
    "# 逐层量化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "27ab253a-6cbb-4da5-aa44-d2f9643c6ef5",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/miniconda3/envs/lm_ai_learn/lib/python3.10/site-packages/huggingface_hub/file_download.py:945: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
      "  warnings.warn(\n",
      "CUDA extension not installed.\n",
      "CUDA extension not installed.\n",
      "/root/miniconda3/envs/lm_ai_learn/lib/python3.10/site-packages/huggingface_hub/file_download.py:945: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3701ffd3d5f9479691c4e03783a9cbcb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing model.decoder.layers blocks :   0%|          | 0/24 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4855cebe30354d36b9ca9d7a7ec06242",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1491ae80924646119e457d283c18ad6b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "af7d34e12d744aaebd8f07f4f51a1508",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "13344390fdcb4ad2a80d6585f668b399",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a6c5001f66a04e118dab3df02c021da0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f57ced5dd8ae49308651d99fbeaaaa05",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f27500d104d747e8b53aa2bee855bb9c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b08e30f4ad2a4ccfaffde0f7530f8e6a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "88b6a05450e04f5bbc008c5889f0c418",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c8549e69146645c0ba735935ba578c0d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "658b1b24b53e488b9e8ff8ff292eccc5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "42f9f99125864962a302bfd75651b89b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "00de50ca78ce49529895945f709c47f4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f2c41c5e1419467ba930c8d76e94690e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2db60cca4772403697d12e15a3bd5378",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f0f8d4d16b124de19f12418577891171",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c04ca84b96004c63b92d030459fceaf1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c6d976eda8ab4223ab40b4d50945fe0d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "30c49dd0f8a145aa8a0f927ef2102c1c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a4ee4b10ca3d486ea52992f142b0d86c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fd7f2101aba647a0b567007d71acdebf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c4cc8924899a4286bd2970c659b12979",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "07dc179ef4864618a95aa0929d367ce7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dcfc6e38a5e44a368af8ce26e4e58fd0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "quant_model = AutoModelForCausalLM.from_pretrained(\n",
    "    model_name_or_path,\n",
    "    quantization_config=quantization_config,\n",
    "    device_map='auto')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2ce1e723-7639-4471-8580-94284313ea39",
   "metadata": {},
   "source": [
    "# 检查量化模型正确性"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e2ced59d-c110-4f0b-8684-daca34bf59cc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'training': True,\n",
       " '_parameters': OrderedDict(),\n",
       " '_buffers': OrderedDict([('qweight',\n",
       "               tensor([[  788511998,  -933378217,  -809796260,  ...,   -31381160,\n",
       "                         -436374318, -1217214636],\n",
       "                       [  -11054626,  1109906354,  2100032625,  ...,  -574685957,\n",
       "                         -125641532,   309793127],\n",
       "                       [ -420349487, -1086242979,  -136790040,  ...,  -268966537,\n",
       "                         -595691201,   -20738226],\n",
       "                       ...,\n",
       "                       [-1120454330,  -980290633,  -738222471,  ..., -1785756297,\n",
       "                         1496129148,  -845390851],\n",
       "                       [  937842589,  -851933799, -1309040815,  ...,  -231780748,\n",
       "                         1575547337,  -572076807],\n",
       "                       [ -295838737,  -131861042, -1773611281,  ...,  -102962817,\n",
       "                         -166031573,  -272487427]], device='cuda:0', dtype=torch.int32)),\n",
       "              ('qzeros',\n",
       "               tensor([[1431655765, 1431655765, 1431655765,  ..., 1431655765, 1431655765,\n",
       "                        1431655765],\n",
       "                       [1431655765, 1431655765, 1431655765,  ..., 1431655765, 1431655765,\n",
       "                        1431655765],\n",
       "                       [1431655765, 1431655765, 1431655765,  ..., 1431655765, 1431655765,\n",
       "                        1431655765],\n",
       "                       ...,\n",
       "                       [1431655765, 1431655765, 1431655765,  ..., 1431655765, 1431655765,\n",
       "                        1431655765],\n",
       "                       [1431655765, 1431655765, 1431655765,  ..., 1431655765, 1431655765,\n",
       "                        1431655765],\n",
       "                       [1431655765, 1431655765, 1431655765,  ..., 1431655765, 1431655765,\n",
       "                        1431655765]], device='cuda:0', dtype=torch.int32)),\n",
       "              ('scales',\n",
       "               tensor([[0.0023, 0.0061, 0.0018,  ..., 0.0031, 0.0079, 0.0089],\n",
       "                       [0.0021, 0.0042, 0.0078,  ..., 0.0126, 0.0133, 0.0150],\n",
       "                       [0.0136, 0.0105, 0.0064,  ..., 0.0102, 0.0070, 0.0092],\n",
       "                       ...,\n",
       "                       [0.0126, 0.0076, 0.0068,  ..., 0.0148, 0.0078, 0.0008],\n",
       "                       [0.0103, 0.0070, 0.0126,  ..., 0.0103, 0.0106, 0.0022],\n",
       "                       [0.0099, 0.0035, 0.0121,  ..., 0.0053, 0.0115, 0.0076]],\n",
       "                      device='cuda:0', dtype=torch.float16)),\n",
       "              ('g_idx',\n",
       "               tensor([   0,    0,    1,  ..., 1022, 1023, 1023], device='cuda:0',\n",
       "                      dtype=torch.int32)),\n",
       "              ('bias',\n",
       "               tensor([-0.0407, -0.0028, -0.0199,  ..., -0.0168, -0.0078, -0.0042],\n",
       "                      device='cuda:0', dtype=torch.float16))]),\n",
       " '_non_persistent_buffers_set': set(),\n",
       " '_backward_pre_hooks': OrderedDict(),\n",
       " '_backward_hooks': OrderedDict(),\n",
       " '_is_full_backward_hook': None,\n",
       " '_forward_hooks': OrderedDict(),\n",
       " '_forward_hooks_with_kwargs': OrderedDict(),\n",
       " '_forward_hooks_always_called': OrderedDict(),\n",
       " '_forward_pre_hooks': OrderedDict(),\n",
       " '_forward_pre_hooks_with_kwargs': OrderedDict(),\n",
       " '_state_dict_hooks': OrderedDict(),\n",
       " '_state_dict_pre_hooks': OrderedDict(),\n",
       " '_load_state_dict_pre_hooks': OrderedDict(),\n",
       " '_load_state_dict_post_hooks': OrderedDict(),\n",
       " '_modules': OrderedDict(),\n",
       " 'infeatures': 2048,\n",
       " 'outfeatures': 2048,\n",
       " 'bits': 2,\n",
       " 'group_size': 2,\n",
       " 'maxq': 3,\n",
       " 'half_indim': 1024,\n",
       " 'use_cuda_fp16': True,\n",
       " 'wf': tensor([[ 0,  2,  4,  6,  8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30]],\n",
       "        dtype=torch.int32),\n",
       " 'kernel_switch_threshold': 128,\n",
       " 'autogptq_cuda_available': False,\n",
       " 'autogptq_cuda': None,\n",
       " 'trainable': False,\n",
       " 'device': device(type='cuda', index=0)}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "quant_model.model.decoder.layers[0].self_attn.q_proj.__dict__"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2ee879c4-0ea7-402a-9a92-7279a72c7f3b",
   "metadata": {},
   "source": [
    "# 保存模型权重"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "8f016173-1c01-4ac1-ba97-e427d2f385b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "quant_model_path=\"models/facebook-opt-1.3b\"\n",
    "quant_model.save_pretrained(quant_model_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1741b2bb-abaa-4d50-a381-b79f7d5d0eaf",
   "metadata": {},
   "source": [
    "# 使用GPU加载模型生成文本"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "47377931-7edc-4eff-8624-e94e6bac2c0c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/miniconda3/envs/lm_ai_learn/lib/python3.10/site-packages/huggingface_hub/file_download.py:945: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Merry Christmas! I'm glad to be glad to be glad to be glad to be. I am glad to be glad to be. I am glad to be. I am glad to be. I am glad to be. I am glad to be. I is glad to be. I is glad to be. I is glad to be. I is\n"
     ]
    }
   ],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(model_name_or_path)\n",
    "\n",
    "text = \"Merry Christmas! I'm glad to\"\n",
    "inputs = tokenizer(text, return_tensors=\"pt\").to(0)\n",
    "\n",
    "out = quant_model.generate(**inputs, max_new_tokens=64)\n",
    "print(tokenizer.decode(out[0], skip_special_tokens=True))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "86d54956-9ffb-4871-97ea-ee7e6f3eff37",
   "metadata": {},
   "source": [
    "# 使用自定义数据集量化模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "5fd82d66-aa90-49a6-ba90-ebc4cf8661ec",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "60765f78758a432d9e2094abacf579e1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing model.decoder.layers blocks :   0%|          | 0/24 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c5c356fb06f14bf08cea5fca52043b26",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e903069f58da43a78ac1732bd3eabaa4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "be0b07306bca4496a3ce7299d3e0b368",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "91122e11fe244890897900b25aa657e9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d6d850bcccc0465b9b8afa9f23dc7f71",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "36c55a153fdf4701aac16777cd8eaea5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3ea930604ca046838a66ebdca2729cd3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "45c808b1f76841d29373215d8ce841b7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "af0a37c3f6e8461fb5261ff1ab787ec6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "52d4a8b3607443548f9d78d9c471d573",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9d85f3641f4f45a689fe1c1f74c8579f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8ebb565fe07f4680830470802152fcdd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fa0c1a2fa95b427bb1c3e5ef026db736",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "57f0c2a97dfa4c11b6a3aec01217d24b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c4d94943f95845f5bf5c621ba50881f7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "11c5578d702e41a395cfc9a6c5f14c67",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "21a5db6b59fc430f8dba7bf9072c6fb9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "83e2d185cb3a47ff9bc274ae03520303",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "510fa4b3acda43c1af39146b1b4e1c23",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6e70573ccda14b4f9d9ee04506eec510",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7fe1e3a213d441048ca682b05a09f232",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "53eea0514b5e4316b13fb1b1787c1696",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e68231a8fb5f40b4b4a1cfa1ef30e0e5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7a28857a34d6404da6c2f2b7542532ef",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from transformers import AutoModelForCausalLM, GPTQConfig, AutoTokenizer\n",
    "\n",
    "model_name_or_path = \"facebook/opt-1.3b\"\n",
    "custom_dataset = [\"auto-gptq is an easy-to-use model quantization library with user-friendly apis, based on GPTQ algorithm.\"]\n",
    "\n",
    "custom_quantization_config = GPTQConfig(\n",
    "    bits=4,\n",
    "    group_size=128,\n",
    "    desc_act=False,\n",
    "    dataset=custom_dataset\n",
    ")\n",
    "\n",
    "custom_quant_model = AutoModelForCausalLM.from_pretrained(model_name_or_path,\n",
    "                                                          quantization_config=custom_quantization_config,\n",
    "                                                          torch_dtype=torch.float16,\n",
    "                                                          device_map=\"auto\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "4d3defc2-02a5-45ec-b975-76ad10544f0a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Merry Christmas! I'm glad to be to be to to to to to to to to to to to to to to to to to to to to to to to to to to to to to to to to to to to to to to to to to to to to to to to to to to to to to to to to to to to to to\n"
     ]
    }
   ],
   "source": [
    "text = \"Merry Christmas! I'm glad to\"\n",
    "inputs = tokenizer(text, return_tensors=\"pt\").to(0)\n",
    "\n",
    "out = custom_quant_model.generate(**inputs, max_new_tokens=64)\n",
    "print(tokenizer.decode(out[0], skip_special_tokens=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e7db6b4d-43a5-4244-94f5-8993d144beb1",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
